package ca.uwindsor.cs.deepweb.utility.documentanalysis;

import java.io.FileOutputStream;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashSet;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import org.apache.lucene.index.CorruptIndexException;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.TermEnum;

import ca.uwindsor.cs.deepweb.estimation.RandomWordSelector;

public class WordAndDFExtractorwithDictionaryFilter {

	/**
	 * @param args
	 */
	public static void main(String[] args) {
		if (args.length != 4) {
			System.out
					.println("java -jar TermsAndFrequencyExtractor.jar indexDir outputFile");
			System.exit(0);
		}
		try {
			IndexReader reader = IndexReader.open(args[0]);
			System.out.println("NumDocs = " + reader.numDocs());
			
			ArrayList<Integer> list = new ArrayList<Integer>();
			
			RandomWordSelector rws = new RandomWordSelector(args[3],"UTF-8");
			ArrayList<String> dict = rws.getWordList();
			HashSet<String> filter = new HashSet<String>(dict.size());
			for(String word:dict){
				filter.add(word.toLowerCase());
			}
			System.out.println("Dictionary is successfully loaded!");
			
			Pattern p = Pattern.compile("^[A-Za-z]+$");
			Matcher matcher;
			TermEnum te = reader.terms();
			while (te.next()) {
				matcher = p.matcher(te.term().text());
				if (filter.contains(te.term().text().toLowerCase())) {
					//TermFrequencyCell cell = new TermFrequencyCell(te.term().text(),te.docFreq());
//					line = te.term().text() + "\t" + te.docFreq() + "\n";
//					fp.write(line.getBytes("UTF-8"));
//				te.term().text();
					
					list.add(te.docFreq());
				}
			}
			te.close();
			reader.close();
			System.gc();
			Collections.sort(list);
			Collections.reverse(list);
			
			FileOutputStream fpx = new FileOutputStream(args[1]);
			FileOutputStream fpy = new FileOutputStream(args[2]);
			for (int i=0;i<list.size();i++) {
				Integer cell = list.get(i);
//				fpx.write(Double.toString(Math.log10(i+1)).getBytes());
				fpx.write(Integer.toString(i+1).getBytes());
				fpx.write(" ".getBytes());
				fpx.flush();
				
				fpy.write(Integer.toString(cell.intValue()).getBytes());
				fpy.write(" ".getBytes());
				fpy.flush();
				//line = Math.log10(i+1) + "\t" + Math.log10(cell.intValue()) + "\n";
				
			}
			fpx.close();
			fpy.close();

//			fpx = new FileOutputStream(args[3]);
//			fpy = new FileOutputStream(args[4]);
//			for (int i=0;i<1000;i++) {
//				Integer cell = list.get(i);
////				fpx.write(Double.toString(Math.log10(i+1)).getBytes());
//				fpx.write(Integer.toString(i+1).getBytes());
//				fpx.write(" ".getBytes());
//				fpx.flush();
//				
//				fpy.write(Double.toString(Math.log10(cell.intValue())).getBytes());
//				fpy.write(" ".getBytes());
//				fpy.flush();
//				//line = Math.log10(i+1) + "\t" + Math.log10(cell.intValue()) + "\n";
//				
//			}
//			fpx.close();
//			fpy.close();
			
		} catch (CorruptIndexException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		} catch (IOException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		}

	}

}
